blob: 16e93370e5124794568ae28aaac2750861115147 [file] [log] [blame]
Junio C Hamano944ce252018-05-30 22:25:261<?xml version="1.0" encoding="UTF-8"?>
Junio C Hamanof2b74942012-11-20 21:06:262<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
3 "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
4<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">
5<head>
Junio C Hamano9d971152012-12-19 00:43:116<meta http-equiv="Content-Type" content="application/xhtml+xml; charset=UTF-8" />
Junio C Hamano944ce252018-05-30 22:25:267<meta name="generator" content="AsciiDoc 8.6.10" />
Junio C Hamanobc8d4782014-01-13 23:35:508<title>Concerning Git&#8217;s Packing Heuristics</title>
Junio C Hamanof2b74942012-11-20 21:06:269<style type="text/css">
Junio C Hamano9d971152012-12-19 00:43:1110/* Shared CSS for AsciiDoc xhtml11 and html5 backends */
11
12/* Default font. */
13body {
14 font-family: Georgia,serif;
15}
16
17/* Title font. */
18h1, h2, h3, h4, h5, h6,
19div.title, caption.title,
20thead, p.table.header,
21#toctitle,
22#author, #revnumber, #revdate, #revremark,
23#footer {
24 font-family: Arial,Helvetica,sans-serif;
Junio C Hamanof2b74942012-11-20 21:06:2625}
26
27body {
28 margin: 1em 5% 1em 5%;
29}
30
31a {
32 color: blue;
33 text-decoration: underline;
34}
35a:visited {
36 color: fuchsia;
37}
38
39em {
40 font-style: italic;
41 color: navy;
42}
43
44strong {
45 font-weight: bold;
46 color: #083194;
47}
48
Junio C Hamanof2b74942012-11-20 21:06:2649h1, h2, h3, h4, h5, h6 {
50 color: #527bbd;
Junio C Hamanof2b74942012-11-20 21:06:2651 margin-top: 1.2em;
52 margin-bottom: 0.5em;
53 line-height: 1.3;
54}
55
56h1, h2, h3 {
57 border-bottom: 2px solid silver;
58}
59h2 {
60 padding-top: 0.5em;
61}
62h3 {
63 float: left;
64}
65h3 + * {
66 clear: left;
67}
Junio C Hamano9d971152012-12-19 00:43:1168h5 {
69 font-size: 1.0em;
70}
Junio C Hamanof2b74942012-11-20 21:06:2671
72div.sectionbody {
Junio C Hamanof2b74942012-11-20 21:06:2673 margin-left: 0;
74}
75
76hr {
77 border: 1px solid silver;
78}
79
80p {
81 margin-top: 0.5em;
82 margin-bottom: 0.5em;
83}
84
85ul, ol, li > p {
86 margin-top: 0;
87}
Junio C Hamano9d971152012-12-19 00:43:1188ul > li { color: #aaa; }
89ul > li > * { color: black; }
Junio C Hamanof2b74942012-11-20 21:06:2690
Junio C Hamanoc14e6ad2014-10-31 20:25:5391.monospaced, code, pre {
92 font-family: "Courier New", Courier, monospace;
93 font-size: inherit;
94 color: navy;
Junio C Hamanof2b74942012-11-20 21:06:2695 padding: 0;
96 margin: 0;
97}
Junio C Hamanoc14e6ad2014-10-31 20:25:5398pre {
99 white-space: pre-wrap;
100}
Junio C Hamanof2b74942012-11-20 21:06:26101
Junio C Hamano9d971152012-12-19 00:43:11102#author {
Junio C Hamanof2b74942012-11-20 21:06:26103 color: #527bbd;
Junio C Hamanof2b74942012-11-20 21:06:26104 font-weight: bold;
105 font-size: 1.1em;
106}
Junio C Hamano9d971152012-12-19 00:43:11107#email {
Junio C Hamanof2b74942012-11-20 21:06:26108}
Junio C Hamano9d971152012-12-19 00:43:11109#revnumber, #revdate, #revremark {
Junio C Hamanof2b74942012-11-20 21:06:26110}
111
Junio C Hamano9d971152012-12-19 00:43:11112#footer {
Junio C Hamanof2b74942012-11-20 21:06:26113 font-size: small;
114 border-top: 2px solid silver;
115 padding-top: 0.5em;
116 margin-top: 4.0em;
117}
Junio C Hamano9d971152012-12-19 00:43:11118#footer-text {
Junio C Hamanof2b74942012-11-20 21:06:26119 float: left;
120 padding-bottom: 0.5em;
121}
Junio C Hamano9d971152012-12-19 00:43:11122#footer-badges {
Junio C Hamanof2b74942012-11-20 21:06:26123 float: right;
124 padding-bottom: 0.5em;
125}
126
Junio C Hamano9d971152012-12-19 00:43:11127#preamble {
Junio C Hamanof2b74942012-11-20 21:06:26128 margin-top: 1.5em;
129 margin-bottom: 1.5em;
130}
Junio C Hamano9d971152012-12-19 00:43:11131div.imageblock, div.exampleblock, div.verseblock,
Junio C Hamanof2b74942012-11-20 21:06:26132div.quoteblock, div.literalblock, div.listingblock, div.sidebarblock,
133div.admonitionblock {
134 margin-top: 1.0em;
135 margin-bottom: 1.5em;
136}
137div.admonitionblock {
138 margin-top: 2.0em;
139 margin-bottom: 2.0em;
140 margin-right: 10%;
141 color: #606060;
142}
143
144div.content { /* Block element content. */
145 padding: 0;
146}
147
148/* Block element titles. */
149div.title, caption.title {
150 color: #527bbd;
Junio C Hamanof2b74942012-11-20 21:06:26151 font-weight: bold;
152 text-align: left;
153 margin-top: 1.0em;
154 margin-bottom: 0.5em;
155}
156div.title + * {
157 margin-top: 0;
158}
159
160td div.title:first-child {
161 margin-top: 0.0em;
162}
163div.content div.title:first-child {
164 margin-top: 0.0em;
165}
166div.content + div.title {
167 margin-top: 0.0em;
168}
169
170div.sidebarblock > div.content {
171 background: #ffffee;
Junio C Hamano9d971152012-12-19 00:43:11172 border: 1px solid #dddddd;
173 border-left: 4px solid #f0f0f0;
Junio C Hamanof2b74942012-11-20 21:06:26174 padding: 0.5em;
175}
176
177div.listingblock > div.content {
Junio C Hamano9d971152012-12-19 00:43:11178 border: 1px solid #dddddd;
179 border-left: 5px solid #f0f0f0;
180 background: #f8f8f8;
Junio C Hamanof2b74942012-11-20 21:06:26181 padding: 0.5em;
182}
183
184div.quoteblock, div.verseblock {
185 padding-left: 1.0em;
186 margin-left: 1.0em;
187 margin-right: 10%;
Junio C Hamano9d971152012-12-19 00:43:11188 border-left: 5px solid #f0f0f0;
189 color: #888;
Junio C Hamanof2b74942012-11-20 21:06:26190}
191
192div.quoteblock > div.attribution {
193 padding-top: 0.5em;
194 text-align: right;
195}
196
Junio C Hamano9d971152012-12-19 00:43:11197div.verseblock > pre.content {
198 font-family: inherit;
199 font-size: inherit;
Junio C Hamanof2b74942012-11-20 21:06:26200}
201div.verseblock > div.attribution {
202 padding-top: 0.75em;
203 text-align: left;
204}
205/* DEPRECATED: Pre version 8.2.7 verse style literal block. */
206div.verseblock + div.attribution {
207 text-align: left;
208}
209
210div.admonitionblock .icon {
211 vertical-align: top;
212 font-size: 1.1em;
213 font-weight: bold;
214 text-decoration: underline;
215 color: #527bbd;
216 padding-right: 0.5em;
217}
218div.admonitionblock td.content {
219 padding-left: 0.5em;
220 border-left: 3px solid #dddddd;
221}
222
223div.exampleblock > div.content {
224 border-left: 3px solid #dddddd;
225 padding-left: 0.5em;
226}
227
228div.imageblock div.content { padding-left: 0; }
Junio C Hamanoc14e6ad2014-10-31 20:25:53229span.image img { border-style: none; vertical-align: text-bottom; }
Junio C Hamanof2b74942012-11-20 21:06:26230a.image:visited { color: white; }
231
232dl {
233 margin-top: 0.8em;
234 margin-bottom: 0.8em;
235}
236dt {
237 margin-top: 0.5em;
238 margin-bottom: 0;
239 font-style: normal;
240 color: navy;
241}
242dd > *:first-child {
243 margin-top: 0.1em;
244}
245
246ul, ol {
247 list-style-position: outside;
248}
249ol.arabic {
250 list-style-type: decimal;
251}
252ol.loweralpha {
253 list-style-type: lower-alpha;
254}
255ol.upperalpha {
256 list-style-type: upper-alpha;
257}
258ol.lowerroman {
259 list-style-type: lower-roman;
260}
261ol.upperroman {
262 list-style-type: upper-roman;
263}
264
265div.compact ul, div.compact ol,
266div.compact p, div.compact p,
267div.compact div, div.compact div {
268 margin-top: 0.1em;
269 margin-bottom: 0.1em;
270}
271
Junio C Hamanof2b74942012-11-20 21:06:26272tfoot {
273 font-weight: bold;
274}
275td > div.verse {
276 white-space: pre;
277}
Junio C Hamanof2b74942012-11-20 21:06:26278
279div.hdlist {
280 margin-top: 0.8em;
281 margin-bottom: 0.8em;
282}
283div.hdlist tr {
284 padding-bottom: 15px;
285}
286dt.hdlist1.strong, td.hdlist1.strong {
287 font-weight: bold;
288}
289td.hdlist1 {
290 vertical-align: top;
291 font-style: normal;
292 padding-right: 0.8em;
293 color: navy;
294}
295td.hdlist2 {
296 vertical-align: top;
297}
298div.hdlist.compact tr {
299 margin: 0;
300 padding-bottom: 0;
301}
302
303.comment {
304 background: yellow;
305}
306
307.footnote, .footnoteref {
308 font-size: 0.8em;
309}
310
311span.footnote, span.footnoteref {
312 vertical-align: super;
313}
314
315#footnotes {
316 margin: 20px 0 20px 0;
317 padding: 7px 0 0 0;
318}
319
320#footnotes div.footnote {
321 margin: 0 0 5px 0;
322}
323
324#footnotes hr {
325 border: none;
326 border-top: 1px solid silver;
327 height: 1px;
328 text-align: left;
329 margin-left: 0;
330 width: 20%;
331 min-width: 100px;
332}
333
Junio C Hamano9d971152012-12-19 00:43:11334div.colist td {
335 padding-right: 0.5em;
336 padding-bottom: 0.3em;
337 vertical-align: top;
338}
339div.colist td img {
340 margin-top: 0.3em;
Junio C Hamanof2b74942012-11-20 21:06:26341}
342
Junio C Hamano9d971152012-12-19 00:43:11343@media print {
344 #footer-badges { display: none; }
345}
346
347#toc {
Junio C Hamanof2b74942012-11-20 21:06:26348 margin-bottom: 2.5em;
349}
350
Junio C Hamano9d971152012-12-19 00:43:11351#toctitle {
Junio C Hamanof2b74942012-11-20 21:06:26352 color: #527bbd;
Junio C Hamanof2b74942012-11-20 21:06:26353 font-size: 1.1em;
354 font-weight: bold;
355 margin-top: 1.0em;
356 margin-bottom: 0.1em;
357}
358
Junio C Hamanoc14e6ad2014-10-31 20:25:53359div.toclevel0, div.toclevel1, div.toclevel2, div.toclevel3, div.toclevel4 {
Junio C Hamanof2b74942012-11-20 21:06:26360 margin-top: 0;
361 margin-bottom: 0;
362}
363div.toclevel2 {
364 margin-left: 2em;
365 font-size: 0.9em;
366}
367div.toclevel3 {
368 margin-left: 4em;
369 font-size: 0.9em;
370}
371div.toclevel4 {
372 margin-left: 6em;
373 font-size: 0.9em;
374}
Junio C Hamanof2b74942012-11-20 21:06:26375
Junio C Hamano9d971152012-12-19 00:43:11376span.aqua { color: aqua; }
377span.black { color: black; }
378span.blue { color: blue; }
379span.fuchsia { color: fuchsia; }
380span.gray { color: gray; }
381span.green { color: green; }
382span.lime { color: lime; }
383span.maroon { color: maroon; }
384span.navy { color: navy; }
385span.olive { color: olive; }
386span.purple { color: purple; }
387span.red { color: red; }
388span.silver { color: silver; }
389span.teal { color: teal; }
390span.white { color: white; }
391span.yellow { color: yellow; }
392
393span.aqua-background { background: aqua; }
394span.black-background { background: black; }
395span.blue-background { background: blue; }
396span.fuchsia-background { background: fuchsia; }
397span.gray-background { background: gray; }
398span.green-background { background: green; }
399span.lime-background { background: lime; }
400span.maroon-background { background: maroon; }
401span.navy-background { background: navy; }
402span.olive-background { background: olive; }
403span.purple-background { background: purple; }
404span.red-background { background: red; }
405span.silver-background { background: silver; }
406span.teal-background { background: teal; }
407span.white-background { background: white; }
408span.yellow-background { background: yellow; }
409
410span.big { font-size: 2em; }
411span.small { font-size: 0.6em; }
412
413span.underline { text-decoration: underline; }
414span.overline { text-decoration: overline; }
415span.line-through { text-decoration: line-through; }
416
Junio C Hamanoc14e6ad2014-10-31 20:25:53417div.unbreakable { page-break-inside: avoid; }
418
Junio C Hamano9d971152012-12-19 00:43:11419
420/*
421 * xhtml11 specific
422 *
423 * */
424
425div.tableblock {
426 margin-top: 1.0em;
427 margin-bottom: 1.5em;
Junio C Hamanof2b74942012-11-20 21:06:26428}
Junio C Hamano9d971152012-12-19 00:43:11429div.tableblock > table {
430 border: 3px solid #527bbd;
431}
432thead, p.table.header {
Junio C Hamanof2b74942012-11-20 21:06:26433 font-weight: bold;
Junio C Hamano9d971152012-12-19 00:43:11434 color: #527bbd;
435}
436p.table {
437 margin-top: 0;
438}
439/* Because the table frame attribute is overriden by CSS in most browsers. */
440div.tableblock > table[frame="void"] {
441 border-style: none;
442}
443div.tableblock > table[frame="hsides"] {
444 border-left-style: none;
445 border-right-style: none;
446}
447div.tableblock > table[frame="vsides"] {
448 border-top-style: none;
449 border-bottom-style: none;
Junio C Hamanof2b74942012-11-20 21:06:26450}
451
Junio C Hamano9d971152012-12-19 00:43:11452
453/*
454 * html5 specific
455 *
456 * */
457
458table.tableblock {
459 margin-top: 1.0em;
460 margin-bottom: 1.5em;
461}
462thead, p.tableblock.header {
463 font-weight: bold;
464 color: #527bbd;
465}
466p.tableblock {
467 margin-top: 0;
468}
469table.tableblock {
470 border-width: 3px;
471 border-spacing: 0px;
472 border-style: solid;
473 border-color: #527bbd;
474 border-collapse: collapse;
475}
476th.tableblock, td.tableblock {
477 border-width: 1px;
478 padding: 4px;
479 border-style: solid;
480 border-color: #527bbd;
Junio C Hamanof2b74942012-11-20 21:06:26481}
482
Junio C Hamano9d971152012-12-19 00:43:11483table.tableblock.frame-topbot {
484 border-left-style: hidden;
485 border-right-style: hidden;
486}
487table.tableblock.frame-sides {
488 border-top-style: hidden;
489 border-bottom-style: hidden;
490}
491table.tableblock.frame-none {
492 border-style: hidden;
493}
494
495th.tableblock.halign-left, td.tableblock.halign-left {
496 text-align: left;
497}
498th.tableblock.halign-center, td.tableblock.halign-center {
499 text-align: center;
500}
501th.tableblock.halign-right, td.tableblock.halign-right {
Junio C Hamanof2b74942012-11-20 21:06:26502 text-align: right;
503}
504
Junio C Hamano9d971152012-12-19 00:43:11505th.tableblock.valign-top, td.tableblock.valign-top {
506 vertical-align: top;
Junio C Hamanof2b74942012-11-20 21:06:26507}
Junio C Hamano9d971152012-12-19 00:43:11508th.tableblock.valign-middle, td.tableblock.valign-middle {
509 vertical-align: middle;
510}
511th.tableblock.valign-bottom, td.tableblock.valign-bottom {
512 vertical-align: bottom;
Junio C Hamanof2b74942012-11-20 21:06:26513}
514
Junio C Hamano9d971152012-12-19 00:43:11515
516/*
517 * manpage specific
518 *
519 * */
520
521body.manpage h1 {
522 padding-top: 0.5em;
523 padding-bottom: 0.5em;
524 border-top: 2px solid silver;
525 border-bottom: 2px solid silver;
526}
527body.manpage h2 {
528 border-style: none;
529}
530body.manpage div.sectionbody {
531 margin-left: 3em;
Junio C Hamanof2b74942012-11-20 21:06:26532}
533
Junio C Hamano9d971152012-12-19 00:43:11534@media print {
535 body.manpage div#toc { display: none; }
536}
Junio C Hamanoc14e6ad2014-10-31 20:25:53537
538
Junio C Hamanof2b74942012-11-20 21:06:26539</style>
540<script type="text/javascript">
541/*<![CDATA[*/
Junio C Hamanof2b74942012-11-20 21:06:26542var asciidoc = { // Namespace.
543
544/////////////////////////////////////////////////////////////////////
545// Table Of Contents generator
546/////////////////////////////////////////////////////////////////////
547
548/* Author: Mihai Bazon, September 2002
549 * http://students.infoiasi.ro/~mishoo
550 *
551 * Table Of Content generator
552 * Version: 0.4
553 *
554 * Feel free to use this script under the terms of the GNU General Public
555 * License, as long as you do not remove or alter this notice.
556 */
557
558 /* modified by Troy D. Hanson, September 2006. License: GPL */
559 /* modified by Stuart Rackham, 2006, 2009. License: GPL */
560
561// toclevels = 1..4.
562toc: function (toclevels) {
563
564 function getText(el) {
565 var text = "";
566 for (var i = el.firstChild; i != null; i = i.nextSibling) {
567 if (i.nodeType == 3 /* Node.TEXT_NODE */) // IE doesn't speak constants.
568 text += i.data;
569 else if (i.firstChild != null)
570 text += getText(i);
571 }
572 return text;
573 }
574
575 function TocEntry(el, text, toclevel) {
576 this.element = el;
577 this.text = text;
578 this.toclevel = toclevel;
579 }
580
581 function tocEntries(el, toclevels) {
582 var result = new Array;
Junio C Hamanoc14e6ad2014-10-31 20:25:53583 var re = new RegExp('[hH]([1-'+(toclevels+1)+'])');
Junio C Hamanof2b74942012-11-20 21:06:26584 // Function that scans the DOM tree for header elements (the DOM2
585 // nodeIterator API would be a better technique but not supported by all
586 // browsers).
587 var iterate = function (el) {
588 for (var i = el.firstChild; i != null; i = i.nextSibling) {
589 if (i.nodeType == 1 /* Node.ELEMENT_NODE */) {
590 var mo = re.exec(i.tagName);
591 if (mo && (i.getAttribute("class") || i.getAttribute("className")) != "float") {
592 result[result.length] = new TocEntry(i, getText(i), mo[1]-1);
593 }
594 iterate(i);
595 }
596 }
597 }
598 iterate(el);
599 return result;
600 }
601
602 var toc = document.getElementById("toc");
Junio C Hamano9d971152012-12-19 00:43:11603 if (!toc) {
604 return;
605 }
606
607 // Delete existing TOC entries in case we're reloading the TOC.
608 var tocEntriesToRemove = [];
609 var i;
610 for (i = 0; i < toc.childNodes.length; i++) {
611 var entry = toc.childNodes[i];
Junio C Hamanoc14e6ad2014-10-31 20:25:53612 if (entry.nodeName.toLowerCase() == 'div'
Junio C Hamano9d971152012-12-19 00:43:11613 && entry.getAttribute("class")
614 && entry.getAttribute("class").match(/^toclevel/))
615 tocEntriesToRemove.push(entry);
616 }
617 for (i = 0; i < tocEntriesToRemove.length; i++) {
618 toc.removeChild(tocEntriesToRemove[i]);
619 }
620
621 // Rebuild TOC entries.
Junio C Hamanof2b74942012-11-20 21:06:26622 var entries = tocEntries(document.getElementById("content"), toclevels);
623 for (var i = 0; i < entries.length; ++i) {
624 var entry = entries[i];
625 if (entry.element.id == "")
626 entry.element.id = "_toc_" + i;
627 var a = document.createElement("a");
628 a.href = "#" + entry.element.id;
629 a.appendChild(document.createTextNode(entry.text));
630 var div = document.createElement("div");
631 div.appendChild(a);
632 div.className = "toclevel" + entry.toclevel;
633 toc.appendChild(div);
634 }
635 if (entries.length == 0)
636 toc.parentNode.removeChild(toc);
637},
638
639
640/////////////////////////////////////////////////////////////////////
641// Footnotes generator
642/////////////////////////////////////////////////////////////////////
643
644/* Based on footnote generation code from:
645 * http://www.brandspankingnew.net/archive/2005/07/format_footnote.html
646 */
647
648footnotes: function () {
Junio C Hamano9d971152012-12-19 00:43:11649 // Delete existing footnote entries in case we're reloading the footnodes.
650 var i;
Junio C Hamanof2b74942012-11-20 21:06:26651 var noteholder = document.getElementById("footnotes");
Junio C Hamano9d971152012-12-19 00:43:11652 if (!noteholder) {
653 return;
654 }
655 var entriesToRemove = [];
656 for (i = 0; i < noteholder.childNodes.length; i++) {
657 var entry = noteholder.childNodes[i];
Junio C Hamanoc14e6ad2014-10-31 20:25:53658 if (entry.nodeName.toLowerCase() == 'div' && entry.getAttribute("class") == "footnote")
Junio C Hamano9d971152012-12-19 00:43:11659 entriesToRemove.push(entry);
660 }
661 for (i = 0; i < entriesToRemove.length; i++) {
662 noteholder.removeChild(entriesToRemove[i]);
663 }
664
665 // Rebuild footnote entries.
666 var cont = document.getElementById("content");
Junio C Hamanof2b74942012-11-20 21:06:26667 var spans = cont.getElementsByTagName("span");
668 var refs = {};
669 var n = 0;
670 for (i=0; i<spans.length; i++) {
671 if (spans[i].className == "footnote") {
672 n++;
Junio C Hamano9d971152012-12-19 00:43:11673 var note = spans[i].getAttribute("data-note");
674 if (!note) {
675 // Use [\s\S] in place of . so multi-line matches work.
676 // Because JavaScript has no s (dotall) regex flag.
677 note = spans[i].innerHTML.match(/\s*\[([\s\S]*)]\s*/)[1];
678 spans[i].innerHTML =
679 "[<a id='_footnoteref_" + n + "' href='#_footnote_" + n +
680 "' title='View footnote' class='footnote'>" + n + "</a>]";
681 spans[i].setAttribute("data-note", note);
682 }
Junio C Hamanof2b74942012-11-20 21:06:26683 noteholder.innerHTML +=
684 "<div class='footnote' id='_footnote_" + n + "'>" +
685 "<a href='#_footnoteref_" + n + "' title='Return to text'>" +
686 n + "</a>. " + note + "</div>";
Junio C Hamanof2b74942012-11-20 21:06:26687 var id =spans[i].getAttribute("id");
688 if (id != null) refs["#"+id] = n;
689 }
690 }
691 if (n == 0)
692 noteholder.parentNode.removeChild(noteholder);
693 else {
694 // Process footnoterefs.
695 for (i=0; i<spans.length; i++) {
696 if (spans[i].className == "footnoteref") {
697 var href = spans[i].getElementsByTagName("a")[0].getAttribute("href");
698 href = href.match(/#.*/)[0]; // Because IE return full URL.
699 n = refs[href];
700 spans[i].innerHTML =
701 "[<a href='#_footnote_" + n +
702 "' title='View footnote' class='footnote'>" + n + "</a>]";
703 }
704 }
705 }
Junio C Hamano9d971152012-12-19 00:43:11706},
707
708install: function(toclevels) {
709 var timerId;
710
711 function reinstall() {
712 asciidoc.footnotes();
713 if (toclevels) {
714 asciidoc.toc(toclevels);
715 }
716 }
717
718 function reinstallAndRemoveTimer() {
719 clearInterval(timerId);
720 reinstall();
721 }
722
723 timerId = setInterval(reinstall, 500);
724 if (document.addEventListener)
725 document.addEventListener("DOMContentLoaded", reinstallAndRemoveTimer, false);
726 else
727 window.onload = reinstallAndRemoveTimer;
Junio C Hamanof2b74942012-11-20 21:06:26728}
729
730}
Junio C Hamano9d971152012-12-19 00:43:11731asciidoc.install();
Junio C Hamanof2b74942012-11-20 21:06:26732/*]]>*/
733</script>
734</head>
Junio C Hamano9d971152012-12-19 00:43:11735<body class="article">
Junio C Hamanof2b74942012-11-20 21:06:26736<div id="header">
Junio C Hamanobc8d4782014-01-13 23:35:50737<h1>Concerning Git&#8217;s Packing Heuristics</h1>
Junio C Hamanof2b74942012-11-20 21:06:26738</div>
739<div id="content">
Junio C Hamanobc8d4782014-01-13 23:35:50740<div id="preamble">
741<div class="sectionbody">
Junio C Hamanof2b74942012-11-20 21:06:26742<div class="literalblock">
743<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:53744<pre><code>Oh, here's a really stupid question:</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:26745</div></div>
746<div class="literalblock">
747<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:53748<pre><code> Where do I go
Junio C Hamanof2b74942012-11-20 21:06:26749 to learn the details
Junio C Hamanoc14e6ad2014-10-31 20:25:53750of Git's packing heuristics?</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:26751</div></div>
752<div class="paragraph"><p>Be careful what you ask!</p></div>
Junio C Hamano076ffcc2013-02-06 05:13:21753<div class="paragraph"><p>Followers of the Git, please open the Git IRC Log and turn to
Junio C Hamanof2b74942012-11-20 21:06:26754February 10, 2006.</p></div>
755<div class="paragraph"><p>It&#8217;s a rare occasion, and we are joined by the King Git Himself,
756Linus Torvalds (linus). Nathaniel Smith, (njs`), has the floor
757and seeks enlightenment. Others are present, but silent.</p></div>
758<div class="paragraph"><p>Let&#8217;s listen in!</p></div>
759<div class="literalblock">
760<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:53761<pre><code>&lt;njs`&gt; Oh, here's a really stupid question -- where do I go to
Junio C Hamano076ffcc2013-02-06 05:13:21762 learn the details of Git's packing heuristics? google avails
Junio C Hamanof2b74942012-11-20 21:06:26763 me not, reading the source didn't help a lot, and wading
764 through the whole mailing list seems less efficient than any
Junio C Hamanoc14e6ad2014-10-31 20:25:53765 of that.</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:26766</div></div>
767<div class="paragraph"><p>It is a bold start! A plea for help combined with a simultaneous
768tri-part attack on some of the tried and true mainstays in the quest
769for enlightenment. Brash accusations of google being useless. Hubris!
770Maligning the source. Heresy! Disdain for the mailing list archives.
771Woe.</p></div>
772<div class="literalblock">
773<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:53774<pre><code>&lt;pasky&gt; yes, the packing-related delta stuff is somewhat
775 mysterious even for me ;)</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:26776</div></div>
777<div class="paragraph"><p>Ah! Modesty after all.</p></div>
778<div class="literalblock">
779<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:53780<pre><code>&lt;linus&gt; njs, I don't think the docs exist. That's something where
Junio C Hamanof2b74942012-11-20 21:06:26781 I don't think anybody else than me even really got involved.
Junio C Hamano076ffcc2013-02-06 05:13:21782 Most of the rest of Git others have been busy with (especially
Junio C Hamanoc14e6ad2014-10-31 20:25:53783 Junio), but packing nobody touched after I did it.</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:26784</div></div>
785<div class="paragraph"><p>It&#8217;s cryptic, yet vague. Linus in style for sure. Wise men
786interpret this as an apology. A few argue it is merely a
787statement of fact.</p></div>
788<div class="literalblock">
789<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:53790<pre><code>&lt;njs`&gt; I guess the next step is "read the source again", but I
791 have to build up a certain level of gumption first :-)</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:26792</div></div>
793<div class="paragraph"><p>Indeed! On both points.</p></div>
794<div class="literalblock">
795<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:53796<pre><code>&lt;linus&gt; The packing heuristic is actually really really simple.</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:26797</div></div>
798<div class="paragraph"><p>Bait&#8230;</p></div>
799<div class="literalblock">
800<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:53801<pre><code>&lt;linus&gt; But strange.</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:26802</div></div>
803<div class="paragraph"><p>And switch. That ought to do it!</p></div>
804<div class="literalblock">
805<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:53806<pre><code>&lt;linus&gt; Remember: Git really doesn't follow files. So what it does is
Junio C Hamanof2b74942012-11-20 21:06:26807 - generate a list of all objects
808 - sort the list according to magic heuristics
809 - walk the list, using a sliding window, seeing if an object
810 can be diffed against another object in the window
Junio C Hamanoc14e6ad2014-10-31 20:25:53811 - write out the list in recency order</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:26812</div></div>
813<div class="paragraph"><p>The traditional understatement:</p></div>
814<div class="literalblock">
815<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:53816<pre><code>&lt;njs`&gt; I suspect that what I'm missing is the precise definition of
817 the word "magic"</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:26818</div></div>
819<div class="paragraph"><p>The traditional insight:</p></div>
820<div class="literalblock">
821<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:53822<pre><code>&lt;pasky&gt; yes</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:26823</div></div>
824<div class="paragraph"><p>And Babel-like confusion flowed.</p></div>
825<div class="literalblock">
826<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:53827<pre><code>&lt;njs`&gt; oh, hmm, and I'm not sure what this sliding window means either</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:26828</div></div>
829<div class="literalblock">
830<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:53831<pre><code>&lt;pasky&gt; iirc, it appeared to me to be just the sha1 of the object
832 when reading the code casually ...</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:26833</div></div>
834<div class="olist lowerroman"><ol class="lowerroman">
835<li>
836<p>
837which simply doesn&#8217;t sound as a very good heuristics, though ;)
838</p>
839<div class="literalblock">
840<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:53841<pre><code>&lt;njs`&gt; .....and recency order. okay, I think it's clear I didn't
842 even realize how much I wasn't realizing :-)</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:26843</div></div>
844</li>
845</ol></div>
846<div class="paragraph"><p>Ah, grasshopper! And thus the enlightenment begins anew.</p></div>
847<div class="literalblock">
848<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:53849<pre><code>&lt;linus&gt; The "magic" is actually in theory totally arbitrary.
Junio C Hamanof2b74942012-11-20 21:06:26850 ANY order will give you a working pack, but no, it's not
Junio C Hamanoc14e6ad2014-10-31 20:25:53851 ordered by SHA-1.</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:26852</div></div>
853<div class="literalblock">
854<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:53855<pre><code>Before talking about the ordering for the sliding delta
Junio C Hamanof2b74942012-11-20 21:06:26856window, let's talk about the recency order. That's more
Junio C Hamanoc14e6ad2014-10-31 20:25:53857important in one way.</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:26858</div></div>
859<div class="literalblock">
860<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:53861<pre><code>&lt;njs`&gt; Right, but if all you want is a working way to pack things
Junio C Hamanof2b74942012-11-20 21:06:26862 together, you could just use cat and save yourself some
Junio C Hamanoc14e6ad2014-10-31 20:25:53863 trouble...</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:26864</div></div>
865<div class="paragraph"><p>Waaait for it&#8230;.</p></div>
866<div class="literalblock">
867<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:53868<pre><code>&lt;linus&gt; The recency ordering (which is basically: put objects
Junio C Hamanof2b74942012-11-20 21:06:26869 _physically_ into the pack in the order that they are
Junio C Hamanoc14e6ad2014-10-31 20:25:53870 "reachable" from the head) is important.</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:26871</div></div>
872<div class="literalblock">
873<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:53874<pre><code>&lt;njs`&gt; okay</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:26875</div></div>
876<div class="literalblock">
877<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:53878<pre><code>&lt;linus&gt; It's important because that's the thing that gives packs
Junio C Hamanof2b74942012-11-20 21:06:26879 good locality. It keeps the objects close to the head (whether
880 they are old or new, but they are _reachable_ from the head)
881 at the head of the pack. So packs actually have absolutely
Junio C Hamanoc14e6ad2014-10-31 20:25:53882 _wonderful_ IO patterns.</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:26883</div></div>
884<div class="paragraph"><p>Read that again, because it is important.</p></div>
885<div class="literalblock">
886<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:53887<pre><code>&lt;linus&gt; But recency ordering is totally useless for deciding how
Junio C Hamanof2b74942012-11-20 21:06:26888 to actually generate the deltas, so the delta ordering is
Junio C Hamanoc14e6ad2014-10-31 20:25:53889 something else.</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:26890</div></div>
891<div class="literalblock">
892<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:53893<pre><code>The delta ordering is (wait for it):
Junio C Hamanof2b74942012-11-20 21:06:26894- first sort by the "basename" of the object, as defined by
895 the name the object was _first_ reached through when
896 generating the object list
897- within the same basename, sort by size of the object
Junio C Hamanoc14e6ad2014-10-31 20:25:53898- but always sort different types separately (commits first).</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:26899</div></div>
900<div class="literalblock">
901<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:53902<pre><code>That's not exactly it, but it's very close.</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:26903</div></div>
904<div class="literalblock">
905<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:53906<pre><code>&lt;njs`&gt; The "_first_ reached" thing is not too important, just you
Junio C Hamanof2b74942012-11-20 21:06:26907 need some way to break ties since the same objects may be
Junio C Hamanoc14e6ad2014-10-31 20:25:53908 reachable many ways, yes?</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:26909</div></div>
910<div class="paragraph"><p>And as if to clarify:</p></div>
911<div class="literalblock">
912<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:53913<pre><code>&lt;linus&gt; The point is that it's all really just any random
Junio C Hamanof2b74942012-11-20 21:06:26914 heuristic, and the ordering is totally unimportant for
915 correctness, but it helps a lot if the heuristic gives
916 "clumping" for things that are likely to delta well against
Junio C Hamanoc14e6ad2014-10-31 20:25:53917 each other.</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:26918</div></div>
919<div class="paragraph"><p>It is an important point, so secretly, I did my own research and have
920included my results below. To be fair, it has changed some over time.
921And through the magic of Revisionistic History, I draw upon this entry
922from The Git IRC Logs on my father&#8217;s birthday, March 1:</p></div>
923<div class="literalblock">
924<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:53925<pre><code>&lt;gitster&gt; The quote from the above linus should be rewritten a
Junio C Hamanof2b74942012-11-20 21:06:26926 bit (wait for it):
927 - first sort by type. Different objects never delta with
928 each other.
929 - then sort by filename/dirname. hash of the basename
930 occupies the top BITS_PER_INT-DIR_BITS bits, and bottom
931 DIR_BITS are for the hash of leading path elements.
932 - then if we are doing "thin" pack, the objects we are _not_
933 going to pack but we know about are sorted earlier than
934 other objects.
Junio C Hamanoc14e6ad2014-10-31 20:25:53935 - and finally sort by size, larger to smaller.</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:26936</div></div>
937<div class="paragraph"><p>In one swell-foop, clarification and obscurification! Nonetheless,
938authoritative. Cryptic, yet concise. It even solicits notions of
939quotes from The Source Code. Clearly, more study is needed.</p></div>
940<div class="literalblock">
941<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:53942<pre><code>&lt;gitster&gt; That's the sort order. What this means is:
Junio C Hamanof2b74942012-11-20 21:06:26943 - we do not delta different object types.
944 - we prefer to delta the objects with the same full path, but
945 allow files with the same name from different directories.
946 - we always prefer to delta against objects we are not going
947 to send, if there are some.
948 - we prefer to delta against larger objects, so that we have
Junio C Hamanoc14e6ad2014-10-31 20:25:53949 lots of removals.</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:26950</div></div>
951<div class="literalblock">
952<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:53953<pre><code>The penultimate rule is for "thin" packs. It is used when
954the other side is known to have such objects.</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:26955</div></div>
956<div class="paragraph"><p>There it is again. "Thin" packs. I&#8217;m thinking to myself, "What
957is a <em>thin</em> pack?" So I ask:</p></div>
958<div class="literalblock">
959<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:53960<pre><code>&lt;jdl&gt; What is a "thin" pack?</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:26961</div></div>
962<div class="literalblock">
963<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:53964<pre><code>&lt;gitster&gt; Use of --objects-edge to rev-list as the upstream of
965 pack-objects. The pack transfer protocol negotiates that.</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:26966</div></div>
967<div class="paragraph"><p>Woo hoo! Cleared that <em>right</em> up!</p></div>
968<div class="literalblock">
969<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:53970<pre><code>&lt;gitster&gt; There are two directions - push and fetch.</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:26971</div></div>
972<div class="paragraph"><p>There! Did you see it? It is not <em>"push" and "pull"</em>! How often the
973confusion has started here. So casually mentioned, too!</p></div>
974<div class="literalblock">
975<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:53976<pre><code>&lt;gitster&gt; For push, git-send-pack invokes git-receive-pack on the
Junio C Hamanof2b74942012-11-20 21:06:26977 other end. The receive-pack says "I have up to these commits".
978 send-pack looks at them, and computes what are missing from
Junio C Hamanoc14e6ad2014-10-31 20:25:53979 the other end. So "thin" could be the default there.</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:26980</div></div>
981<div class="literalblock">
982<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:53983<pre><code>In the other direction, fetch, git-fetch-pack and
Junio C Hamanof2b74942012-11-20 21:06:26984git-clone-pack invokes git-upload-pack on the other end
Junio C Hamanoc14e6ad2014-10-31 20:25:53985(via ssh or by talking to the daemon).</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:26986</div></div>
987<div class="literalblock">
988<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:53989<pre><code>There are two cases: fetch-pack with -k and clone-pack is one,
Junio C Hamanof2b74942012-11-20 21:06:26990fetch-pack without -k is the other. clone-pack and fetch-pack
991with -k will keep the downloaded packfile without expanded, so
992we do not use thin pack transfer. Otherwise, the generated
Junio C Hamanoc14e6ad2014-10-31 20:25:53993pack will have delta without base object in the same pack.</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:26994</div></div>
995<div class="literalblock">
996<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:53997<pre><code>But fetch-pack without -k will explode the received pack into
Junio C Hamanof2b74942012-11-20 21:06:26998individual objects, so we automatically ask upload-pack to
Junio C Hamanoc14e6ad2014-10-31 20:25:53999give us a thin pack if upload-pack supports it.</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261000</div></div>
1001<div class="paragraph"><p>OK then.</p></div>
1002<div class="paragraph"><p>Uh.</p></div>
1003<div class="paragraph"><p>Let&#8217;s return to the previous conversation still in progress.</p></div>
1004<div class="literalblock">
1005<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531006<pre><code>&lt;njs`&gt; and "basename" means something like "the tail of end of
Junio C Hamanof2b74942012-11-20 21:06:261007 path of file objects and dir objects, as per basename(3), and
1008 we just declare all commit and tag objects to have the same
Junio C Hamanoc14e6ad2014-10-31 20:25:531009 basename" or something?</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261010</div></div>
1011<div class="paragraph"><p>Luckily, that too is a point that gitster clarified for us!</p></div>
1012<div class="paragraph"><p>If I might add, the trick is to make files that <em>might</em> be similar be
1013located close to each other in the hash buckets based on their file
1014names. It used to be that "foo/Makefile", "bar/baz/quux/Makefile" and
1015"Makefile" all landed in the same bucket due to their common basename,
1016"Makefile". However, now they land in "close" buckets.</p></div>
1017<div class="paragraph"><p>The algorithm allows not just for the <em>same</em> bucket, but for <em>close</em>
1018buckets to be considered delta candidates. The rationale is
1019essentially that files, like Makefiles, often have very similar
1020content no matter what directory they live in.</p></div>
1021<div class="literalblock">
1022<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531023<pre><code>&lt;linus&gt; I played around with different delta algorithms, and with
Junio C Hamanof2b74942012-11-20 21:06:261024 making the "delta window" bigger, but having too big of a
1025 sliding window makes it very expensive to generate the pack:
Junio C Hamanoc14e6ad2014-10-31 20:25:531026 you need to compare every object with a _ton_ of other objects.</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261027</div></div>
1028<div class="literalblock">
1029<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531030<pre><code>There are a number of other trivial heuristics too, which
Junio C Hamanof2b74942012-11-20 21:06:261031basically boil down to "don't bother even trying to delta this
1032pair" if we can tell before-hand that the delta isn't worth it
1033(due to size differences, where we can take a previous delta
1034result into account to decide that "ok, no point in trying
Junio C Hamanoc14e6ad2014-10-31 20:25:531035that one, it will be worse").</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261036</div></div>
1037<div class="literalblock">
1038<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531039<pre><code>End result: packing is actually very size efficient. It's
Junio C Hamanof2b74942012-11-20 21:06:261040somewhat CPU-wasteful, but on the other hand, since you're
1041really only supposed to do it maybe once a month (and you can
Junio C Hamanoc14e6ad2014-10-31 20:25:531042do it during the night), nobody really seems to care.</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261043</div></div>
1044<div class="paragraph"><p>Nice Engineering Touch, there. Find when it doesn&#8217;t matter, and
1045proclaim it a non-issue. Good style too!</p></div>
1046<div class="literalblock">
1047<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531048<pre><code>&lt;njs`&gt; So, just to repeat to see if I'm following, we start by
Junio C Hamanof2b74942012-11-20 21:06:261049 getting a list of the objects we want to pack, we sort it by
1050 this heuristic (basically lexicographically on the tuple
Junio C Hamanoc14e6ad2014-10-31 20:25:531051 (type, basename, size)).</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261052</div></div>
1053<div class="literalblock">
1054<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531055<pre><code>Then we walk through this list, and calculate a delta of
Junio C Hamanof2b74942012-11-20 21:06:261056each object against the last n (tunable parameter) objects,
Junio C Hamanoc14e6ad2014-10-31 20:25:531057and pick the smallest of these deltas.</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261058</div></div>
1059<div class="paragraph"><p>Vastly simplified, but the essence is there!</p></div>
1060<div class="literalblock">
1061<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531062<pre><code>&lt;linus&gt; Correct.</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261063</div></div>
1064<div class="literalblock">
1065<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531066<pre><code>&lt;njs`&gt; And then once we have picked a delta or fulltext to
Junio C Hamanof2b74942012-11-20 21:06:261067 represent each object, we re-sort by recency, and write them
Junio C Hamanoc14e6ad2014-10-31 20:25:531068 out in that order.</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261069</div></div>
1070<div class="literalblock">
1071<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531072<pre><code>&lt;linus&gt; Yup. Some other small details:</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261073</div></div>
1074<div class="paragraph"><p>And of course there is the "Other Shoe" Factor too.</p></div>
1075<div class="literalblock">
1076<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531077<pre><code>&lt;linus&gt; - We limit the delta depth to another magic value (right
1078 now both the window and delta depth magic values are just "10")</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261079</div></div>
1080<div class="literalblock">
1081<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531082<pre><code>&lt;njs`&gt; Hrm, my intuition is that you'd end up with really _bad_ IO
Junio C Hamanof2b74942012-11-20 21:06:261083 patterns, because the things you want are near by, but to
1084 actually reconstruct them you may have to jump all over in
Junio C Hamanoc14e6ad2014-10-31 20:25:531085 random ways.</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261086</div></div>
1087<div class="literalblock">
1088<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531089<pre><code>&lt;linus&gt; - When we write out a delta, and we haven't yet written
Junio C Hamanof2b74942012-11-20 21:06:261090 out the object it is a delta against, we write out the base
1091 object first. And no, when we reconstruct them, we actually
1092 get nice IO patterns, because:
1093 - larger objects tend to be "more recent" (Linus' law: files grow)
1094 - we actively try to generate deltas from a larger object to a
1095 smaller one
1096 - this means that the top-of-tree very seldom has deltas
Junio C Hamanoc14e6ad2014-10-31 20:25:531097 (i.e. deltas in _practice_ are "backwards deltas")</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261098</div></div>
1099<div class="paragraph"><p>Again, we should reread that whole paragraph. Not just because
1100Linus has slipped Linus&#8217;s Law in there on us, but because it is
1101important. Let&#8217;s make sure we clarify some of the points here:</p></div>
1102<div class="literalblock">
1103<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531104<pre><code>&lt;njs`&gt; So the point is just that in practice, delta order and
1105 recency order match each other quite well.</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261106</div></div>
1107<div class="literalblock">
1108<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531109<pre><code>&lt;linus&gt; Yes. There's another nice side to this (and yes, it was
Junio C Hamanof2b74942012-11-20 21:06:261110 designed that way ;):
1111 - the reason we generate deltas against the larger object is
Junio C Hamanoc14e6ad2014-10-31 20:25:531112 actually a big space saver too!</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261113</div></div>
1114<div class="literalblock">
1115<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531116<pre><code>&lt;njs`&gt; Hmm, but your last comment (if "we haven't yet written out
Junio C Hamanof2b74942012-11-20 21:06:261117 the object it is a delta against, we write out the base object
1118 first"), seems like it would make these facts mostly
1119 irrelevant because even if in practice you would not have to
1120 wander around much, in fact you just brute-force say that in
Junio C Hamanoc14e6ad2014-10-31 20:25:531121 the cases where you might have to wander, don't do that :-)</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261122</div></div>
1123<div class="literalblock">
1124<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531125<pre><code>&lt;linus&gt; Yes and no. Notice the rule: we only write out the base
Junio C Hamanof2b74942012-11-20 21:06:261126 object first if the delta against it was more recent. That
1127 means that you can actually have deltas that refer to a base
1128 object that is _not_ close to the delta object, but that only
Junio C Hamanoc14e6ad2014-10-31 20:25:531129 happens when the delta is needed to generate an _old_ object.</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261130</div></div>
1131<div class="literalblock">
1132<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531133<pre><code>&lt;linus&gt; See?</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261134</div></div>
1135<div class="paragraph"><p>Yeah, no. I missed that on the first two or three readings myself.</p></div>
1136<div class="literalblock">
1137<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531138<pre><code>&lt;linus&gt; This keeps the front of the pack dense. The front of the
Junio C Hamanof2b74942012-11-20 21:06:261139 pack never contains data that isn't relevant to a "recent"
1140 object. The size optimization comes from our use of xdelta
1141 (but is true for many other delta algorithms): removing data
Junio C Hamanoc14e6ad2014-10-31 20:25:531142 is cheaper (in size) than adding data.</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261143</div></div>
1144<div class="literalblock">
1145<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531146<pre><code>When you remove data, you only need to say "copy bytes n--m".
Junio C Hamanof2b74942012-11-20 21:06:261147In contrast, in a delta that _adds_ data, you have to say "add
Junio C Hamanoc14e6ad2014-10-31 20:25:531148these bytes: 'actual data goes here'"</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261149</div></div>
1150<div class="ulist"><ul>
1151<li>
1152<p>
1153njs` has quit: Read error: 104 (Connection reset by peer)
1154</p>
1155<div class="literalblock">
1156<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531157<pre><code>&lt;linus&gt; Uhhuh. I hope I didn't blow njs` mind.</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261158</div></div>
1159</li>
1160<li>
1161<p>
1162njs` has joined channel #git
1163</p>
1164<div class="literalblock">
1165<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531166<pre><code>&lt;pasky&gt; :)</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261167</div></div>
1168</li>
1169</ul></div>
1170<div class="paragraph"><p>The silent observers are amused. Of course.</p></div>
1171<div class="paragraph"><p>And as if njs` was expected to be omniscient:</p></div>
1172<div class="literalblock">
1173<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531174<pre><code>&lt;linus&gt; njs - did you miss anything?</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261175</div></div>
1176<div class="paragraph"><p>OK, I&#8217;ll spell it out. That&#8217;s Geek Humor. If njs` was not actually
1177connected for a little bit there, how would he know if missed anything
1178while he was disconnected? He&#8217;s a benevolent dictator with a sense of
1179humor! Well noted!</p></div>
1180<div class="literalblock">
1181<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531182<pre><code>&lt;njs`&gt; Stupid router. Or gremlins, or whatever.</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261183</div></div>
1184<div class="paragraph"><p>It&#8217;s a cheap shot at Cisco. Take 'em when you can.</p></div>
1185<div class="literalblock">
1186<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531187<pre><code>&lt;njs`&gt; Yes and no. Notice the rule: we only write out the base
1188 object first if the delta against it was more recent.</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261189</div></div>
1190<div class="literalblock">
1191<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531192<pre><code>I'm getting lost in all these orders, let me re-read :-)
Junio C Hamanof2b74942012-11-20 21:06:261193So the write-out order is from most recent to least recent?
1194(Conceivably it could be the opposite way too, I'm not sure if
1195we've said) though my connection back at home is logging, so I
Junio C Hamanoc14e6ad2014-10-31 20:25:531196can just read what you said there :-)</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261197</div></div>
1198<div class="paragraph"><p>And for those of you paying attention, the Omniscient Trick has just
1199been detailed!</p></div>
1200<div class="literalblock">
1201<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531202<pre><code>&lt;linus&gt; Yes, we always write out most recent first</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261203</div></div>
Junio C Hamanof2b74942012-11-20 21:06:261204<div class="literalblock">
1205<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531206<pre><code>&lt;njs`&gt; And, yeah, I got the part about deeper-in-history stuff
1207 having worse IO characteristics, one sort of doesn't care.</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261208</div></div>
1209<div class="literalblock">
1210<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531211<pre><code>&lt;linus&gt; With the caveat that if the "most recent" needs an older
Junio C Hamanof2b74942012-11-20 21:06:261212 object to delta against (hey, shrinking sometimes does
Junio C Hamanoc14e6ad2014-10-31 20:25:531213 happen), we write out the old object with the delta.</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261214</div></div>
1215<div class="literalblock">
1216<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531217<pre><code>&lt;njs`&gt; (if only it happened more...)</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261218</div></div>
1219<div class="literalblock">
1220<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531221<pre><code>&lt;linus&gt; Anyway, the pack-file could easily be denser still, but
Junio C Hamano076ffcc2013-02-06 05:13:211222 because it's used both for streaming (the Git protocol) and
Junio C Hamanoc14e6ad2014-10-31 20:25:531223 for on-disk, it has a few pessimizations.</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261224</div></div>
1225<div class="paragraph"><p>Actually, it is a made-up word. But it is a made-up word being
1226used as setup for a later optimization, which is a real word:</p></div>
1227<div class="literalblock">
1228<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531229<pre><code>&lt;linus&gt; In particular, while the pack-file is then compressed,
Junio C Hamanof2b74942012-11-20 21:06:261230 it's compressed just one object at a time, so the actual
1231 compression factor is less than it could be in theory. But it
1232 means that it's all nice random-access with a simple index to
Junio C Hamanoc14e6ad2014-10-31 20:25:531233 do "object name-&gt;location in packfile" translation.</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261234</div></div>
1235<div class="literalblock">
1236<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531237<pre><code>&lt;njs`&gt; I'm assuming the real win for delta-ing large-&gt;small is
1238 more homogeneous statistics for gzip to run over?</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261239</div></div>
1240<div class="literalblock">
1241<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531242<pre><code>(You have to put the bytes in one place or another, but
1243putting them in a larger blob wins on compression)</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261244</div></div>
1245<div class="literalblock">
1246<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531247<pre><code>Actually, what is the compression strategy -- each delta
Junio C Hamanof2b74942012-11-20 21:06:261248individually gzipped, the whole file gzipped, somewhere in
Junio C Hamanoc14e6ad2014-10-31 20:25:531249between, no compression at all, ....?</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261250</div></div>
1251<div class="literalblock">
1252<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531253<pre><code>Right.</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261254</div></div>
1255<div class="paragraph"><p>Reality IRC sets in. For example:</p></div>
1256<div class="literalblock">
1257<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531258<pre><code>&lt;pasky&gt; I'll read the rest in the morning, I really have to go
Junio C Hamanof2b74942012-11-20 21:06:261259 sleep or there's no hope whatsoever for me at the today's
Junio C Hamanoc14e6ad2014-10-31 20:25:531260 exam... g'nite all.</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261261</div></div>
1262<div class="paragraph"><p>Heh.</p></div>
1263<div class="literalblock">
1264<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531265<pre><code>&lt;linus&gt; pasky: g'nite</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261266</div></div>
1267<div class="literalblock">
1268<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531269<pre><code>&lt;njs`&gt; pasky: 'luck</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261270</div></div>
1271<div class="literalblock">
1272<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531273<pre><code>&lt;linus&gt; Right: large-&gt;small matters exactly because of compression
Junio C Hamanof2b74942012-11-20 21:06:261274 behaviour. If it was non-compressed, it probably wouldn't make
Junio C Hamanoc14e6ad2014-10-31 20:25:531275 any difference.</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261276</div></div>
1277<div class="literalblock">
1278<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531279<pre><code>&lt;njs`&gt; yeah</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261280</div></div>
1281<div class="literalblock">
1282<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531283<pre><code>&lt;linus&gt; Anyway: I'm not even trying to claim that the pack-files
Junio C Hamanof2b74942012-11-20 21:06:261284 are perfect, but they do tend to have a nice balance of
Junio C Hamanoc14e6ad2014-10-31 20:25:531285 density vs ease-of use.</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261286</div></div>
1287<div class="paragraph"><p>Gasp! OK, saved. That&#8217;s a fair Engineering trade off. Close call!
1288In fact, Linus reflects on some Basic Engineering Fundamentals,
1289design options, etc.</p></div>
1290<div class="literalblock">
1291<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531292<pre><code>&lt;linus&gt; More importantly, they allow Git to still _conceptually_
1293 never deal with deltas at all, and be a "whole object" store.</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261294</div></div>
1295<div class="literalblock">
1296<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531297<pre><code>Which has some problems (we discussed bad huge-file
Junio C Hamano076ffcc2013-02-06 05:13:211298behaviour on the Git lists the other day), but it does mean
1299that the basic Git concepts are really really simple and
Junio C Hamanoc14e6ad2014-10-31 20:25:531300straightforward.</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261301</div></div>
1302<div class="literalblock">
1303<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531304<pre><code>It's all been quite stable.</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261305</div></div>
1306<div class="literalblock">
1307<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531308<pre><code>Which I think is very much a result of having very simple
Junio C Hamanof2b74942012-11-20 21:06:261309basic ideas, so that there's never any confusion about what's
Junio C Hamanoc14e6ad2014-10-31 20:25:531310going on.</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261311</div></div>
1312<div class="literalblock">
1313<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531314<pre><code>Bugs happen, but they are "simple" bugs. And bugs that
Junio C Hamanof2b74942012-11-20 21:06:261315actually get some object store detail wrong are almost always
Junio C Hamanoc14e6ad2014-10-31 20:25:531316so obvious that they never go anywhere.</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261317</div></div>
1318<div class="literalblock">
1319<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531320<pre><code>&lt;njs`&gt; Yeah.</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261321</div></div>
1322<div class="paragraph"><p>Nuff said.</p></div>
1323<div class="literalblock">
1324<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531325<pre><code>&lt;linus&gt; Anyway. I'm off for bed. It's not 6AM here, but I've got
Junio C Hamanof2b74942012-11-20 21:06:261326 three kids, and have to get up early in the morning to send
Junio C Hamanoc14e6ad2014-10-31 20:25:531327 them off. I need my beauty sleep.</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261328</div></div>
1329<div class="literalblock">
1330<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531331<pre><code>&lt;njs`&gt; :-)</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261332</div></div>
1333<div class="literalblock">
1334<div class="content">
Junio C Hamanoc14e6ad2014-10-31 20:25:531335<pre><code>&lt;njs`&gt; appreciate the infodump, I really was failing to find the
1336 details on Git packs :-)</code></pre>
Junio C Hamanof2b74942012-11-20 21:06:261337</div></div>
1338<div class="paragraph"><p>And now you know the rest of the story.</p></div>
1339</div>
Junio C Hamanobc8d4782014-01-13 23:35:501340</div>
1341</div>
Junio C Hamanof2b74942012-11-20 21:06:261342<div id="footnotes"><hr /></div>
1343<div id="footer">
1344<div id="footer-text">
Junio C Hamano2ef0ba32018-01-26 23:13:531345Last updated
Junio C Hamano69bb2b52018-11-18 12:44:261346 2018-01-27 08:11:04 JST
Junio C Hamanof2b74942012-11-20 21:06:261347</div>
1348</div>
1349</body>
1350</html>